Главная Обзор Помощь
Зарегистрироваться Войти
J1X
/
huginn
Следить 2
В избранное 0
Ответвить 0

Просмотр исходного кода
Merge pull request #572 from PetroFeed/dropbox-watch-agent

Dropbox watch agent

Andrew Cantino лет %!s(int64=9): %!d(string=назад)
Родитель
f4478d2be3 f9f9c1061b
Сommit
13bfdc3c03
12 измененных файлов с 354 добавлено и 9 удалено
Показать статистику Diff
  1. 5 2
      .env.example
  2. 4 0
      Gemfile
  3. 8 0
      Gemfile.lock
  4. 2 2
      app/assets/stylesheets/application.css.scss.erb
  5. 35 0
      app/concerns/dropbox_concern.rb
  6. 1 1
      app/helpers/application_helper.rb
  7. 114 0
      app/models/agents/dropbox_watch_agent.rb
  8. 1 3
      app/models/service.rb
  9. 8 1
      config/initializers/devise.rb
  10. 1 0
      config/locales/devise.en.yml
  11. 2 0
      spec/env.test
  12. 173 0
      spec/models/agents/dropbox_watch_agent_spec.rb

+ 5 - 2
.env.example
Просмотреть файл 

@@ -53,8 +53,8 @@ INVITATION_CODE=try-huginn
53 53
54 54 
 # Outgoing email settings.  To use Gmail or Google Apps, put your Google Apps domain or gmail.com
55 55 
 # as the SMTP_DOMAIN and your Gmail username and password as the SMTP_USER_NAME and SMTP_PASSWORD.
56 
-#
57 
-# PLEASE NOTE: In order to enable emails locally (e.g., when not in the production Rails environment),
56 
+#
57 
+# PLEASE NOTE: In order to enable emails locally (e.g., when not in the production Rails environment),
58 58 
 # you must also change config.action_mailer.perform_deliveries in config/environments/development.rb.
59 59
60 60 
 SMTP_DOMAIN=your-domain-here.com
 
@@ -92,6 +92,9 @@ GITHUB_OAUTH_SECRET=
92 92 
 TUMBLR_OAUTH_KEY=
93 93 
 TUMBLR_OAUTH_SECRET=
94 94
95 
+DROPBOX_OAUTH_KEY=
96 
+DROPBOX_OAUTH_SECRET=
97 
+
95 98 
 #############################
96 99 
 #  AWS and Mechanical Turk  #
97 100 
 #############################

+ 4 - 0
Gemfile
Просмотреть файл 

@@ -26,6 +26,10 @@ gem 'omniauth-twitter'
26 26 
 gem 'tumblr_client'
27 27 
 gem 'omniauth-tumblr'
28 28
29 
+# Dropbox Agents
30 
+gem 'dropbox-api'
31 
+gem 'omniauth-dropbox'
32 
+
29 33 
 # Optional Services.
30 34 
 gem 'omniauth-37signals'          # BasecampAgent
31 35 
 # gem 'omniauth-github'

+ 8 - 0
Gemfile.lock
Просмотреть файл 

@@ -95,6 +95,10 @@ GEM
95 95 
     dotenv-deployment (0.0.2)
96 96 
     dotenv-rails (0.11.1)
97 97 
       dotenv (= 0.11.1)
98 
+    dropbox-api (0.4.2)
99 
+      hashie
100 
+      multi_json
101 
+      oauth
98 102 
     em-http-request (1.1.2)
99 103 
       addressable (>= 2.3.4)
100 104 
       cookiejar
 
@@ -206,6 +210,8 @@ GEM
206 210 
     omniauth-37signals (1.0.5)
207 211 
       omniauth (~> 1.0)
208 212 
       omniauth-oauth2 (~> 1.0)
213 
+    omniauth-dropbox (0.2.0)
214 
+      omniauth-oauth (~> 1.0)
209 215 
     omniauth-oauth (1.0.1)
210 216 
       oauth
211 217 
       omniauth (~> 1.0)
 
@@ -417,6 +423,7 @@ DEPENDENCIES
417 423 
   devise (~> 3.2.4)
418 424 
   dotenv-deployment
419 425 
   dotenv-rails
426 
+  dropbox-api
420 427 
   em-http-request (~> 1.1.2)
421 428 
   faraday (~> 0.9.0)
422 429 
   faraday_middleware
 
@@ -443,6 +450,7 @@ DEPENDENCIES
443 450 
   nokogiri (~> 1.6.1)
444 451 
   omniauth
445 452 
   omniauth-37signals
453 
+  omniauth-dropbox
446 454 
   omniauth-tumblr
447 455 
   omniauth-twitter
448 456 
   pg

+ 2 - 2
app/assets/stylesheets/application.css.scss.erb
Просмотреть файл 

@@ -241,8 +241,8 @@ h2 .scenario, a span.label.scenario {
241 241 
   width: 200px;
242 242 
 }
243 243
244 
-$services:            twitter     37signals   github      tumblr;
245 
-$service-colors:      #55acee     #8fc857     #444444     #2c4762;
244 
+$services:            twitter     37signals   github      tumblr      dropbox;
245 
+$service-colors:      #55acee     #8fc857     #444444     #2c4762     #007EE5;
246 246
247 247 
 @mixin services {
248 248 
   @each $service in $services {

+ 35 - 0
app/concerns/dropbox_concern.rb
Просмотреть файл 

@@ -0,0 +1,35 @@
1 
+module DropboxConcern
2 
+  extend ActiveSupport::Concern
3 
+
4 
+  included do
5 
+    include Oauthable
6 
+    valid_oauth_providers :dropbox
7 
+    gem_dependency_check { defined?(Dropbox) && Devise.omniauth_providers.include?(:dropbox) }
8 
+  end
9 
+
10 
+  def dropbox
11 
+    Dropbox::API::Config.app_key = consumer_key
12 
+    Dropbox::API::Config.app_secret = consumer_secret
13 
+    Dropbox::API::Config.mode = 'dropbox'
14 
+    Dropbox::API::Client.new(token: oauth_token, secret: oauth_token_secret)
15 
+  end
16 
+
17 
+  private
18 
+
19 
+  def consumer_key
20 
+    (config = Devise.omniauth_configs[:dropbox]) && config.strategy.consumer_key
21 
+  end
22 
+
23 
+  def consumer_secret
24 
+    (config = Devise.omniauth_configs[:dropbox]) && config.strategy.consumer_secret
25 
+  end
26 
+
27 
+  def oauth_token
28 
+    service && service.token
29 
+  end
30 
+
31 
+  def oauth_token_secret
32 
+    service && service.secret
33 
+  end
34 
+
35 
+end

+ 1 - 1
app/helpers/application_helper.rb
Просмотреть файл 

@@ -43,7 +43,7 @@ module ApplicationHelper
43 43
44 44 
   def omniauth_provider_icon(provider)
45 45 
     case provider.to_sym
46 
-    when :twitter, :tumblr, :github
46 
+    when :twitter, :tumblr, :github, :dropbox
47 47 
       content_tag :i, '', class: "fa fa-#{provider}"
48 48 
     else
49 49 
       content_tag :i, '', class: "fa fa-lock"

+ 114 - 0
app/models/agents/dropbox_watch_agent.rb
Просмотреть файл 

@@ -0,0 +1,114 @@
1 
+module Agents
2 
+  class DropboxWatchAgent < Agent
3 
+    include DropboxConcern
4 
+
5 
+    cannot_receive_events!
6 
+    default_schedule "every_1m"
7 
+
8 
+    description <<-MD
9 
+      #{'## Include the `dropbox-api` and `omniauth-dropbox` gems in your `Gemfile` and set `DROPBOX_OAUTH_KEY` and `DROPBOX_OAUTH_SECRET` in your environment to use Dropbox Agents.' if dependencies_missing?}
10 
+      The _DropboxWatchAgent_ watches the given `dir_to_watch` and emits events with the detected changes.
11 
+    MD
12 
+
13 
+    event_description <<-MD
14 
+      The event payload will contain the following fields:
15 
+
16 
+          {
17 
+            "added": [ {
18 
+              "path": "/path/to/added/file",
19 
+              "rev": "1526952fd5",
20 
+              "modified": "Fri, 10 Oct 2014 19:00:43 +0000"
21 
+            } ],
22 
+            "removed": [ ... ],
23 
+            "updated": [ ... ]
24 
+          }
25 
+    MD
26 
+
27 
+    def default_options
28 
+      {
29 
+        'dir_to_watch' => '/',
30 
+        'expected_update_period_in_days' => 1
31 
+      }
32 
+    end
33 
+
34 
+    def validate_options
35 
+      errors.add(:base, 'The `dir_to_watch` property is required.') unless options['dir_to_watch'].present?
36 
+      errors.add(:base, 'Invalid `expected_update_period_in_days` format.') unless options['expected_update_period_in_days'].present? && is_positive_integer?(options['expected_update_period_in_days'])
37 
+    end
38 
+
39 
+    def working?
40 
+      event_created_within?(interpolated['expected_update_period_in_days']) && !received_event_without_error?
41 
+    end
42 
+
43 
+    def check
44 
+      current_contents = ls(interpolated['dir_to_watch'])
45 
+      diff = DropboxDirDiff.new(previous_contents, current_contents)
46 
+      create_event(payload: diff.to_hash) unless previous_contents.nil? || diff.empty?
47 
+
48 
+      remember(current_contents)
49 
+    end
50 
+
51 
+    private
52 
+
53 
+    def is_positive_integer?(value)
54 
+      Integer(value) >= 0
55 
+    rescue
56 
+      false
57 
+    end
58 
+
59 
+    def ls(dir_to_watch)
60 
+      dropbox.ls(dir_to_watch).map { |entry| slice_json(entry, 'path', 'rev', 'modified') }
61 
+    end
62 
+
63 
+    def slice_json(json, *keys)
64 
+      keys.each_with_object({}){|key, hash| hash[key.to_s] = json[key.to_s]}
65 
+    end
66 
+
67 
+    def previous_contents
68 
+      self.memory['contents']
69 
+    end
70 
+
71 
+    def remember(contents)
72 
+      self.memory['contents'] = contents
73 
+    end
74 
+
75 
+    # == Auxiliary classes ==
76 
+
77 
+    class DropboxDirDiff
78 
+      def initialize(previous, current)
79 
+        @previous, @current = [previous || [], current || []]
80 
+      end
81 
+
82 
+      def empty?
83 
+        (@previous == @current)
84 
+      end
85 
+
86 
+      def to_hash
87 
+        calculate_diff
88 
+        { added: @added, removed: @removed, updated: @updated }
89 
+      end
90 
+
91 
+      private
92 
+
93 
+      def calculate_diff
94 
+        @updated = @current.select do |current_entry|
95 
+          previous_entry = find_by_path(@previous, current_entry['path'])
96 
+          (current_entry != previous_entry) && !previous_entry.nil?
97 
+        end
98 
+
99 
+        updated_entries = @updated + @previous.select do |previous_entry|
100 
+          find_by_path(@updated, previous_entry['path'])
101 
+        end
102 
+
103 
+        @added = @current - @previous - updated_entries
104 
+        @removed = @previous - @current - updated_entries
105 
+      end
106 
+
107 
+      def find_by_path(array, path)
108 
+        array.find { |entry| entry['path'] == path }
109 
+      end
110 
+    end
111 
+
112 
+  end
113 
+
114 
+end

+ 1 - 3
app/models/service.rb
Просмотреть файл 

@@ -59,12 +59,10 @@ class Service < ActiveRecord::Base
59 59
60 60 
   def self.provider_specific_options(omniauth)
61 61 
     case omniauth['provider'].to_sym
62 
-      when :twitter, :github
63 
-        { name: omniauth['info']['nickname'] }
64 62 
       when :'37signals'
65 63 
         { user_id: omniauth['extra']['accounts'][0]['id'], name: omniauth['info']['name'] }
66 64 
       else
67 
-        { name: omniauth['info']['nickname'] }
65 
+        { name: omniauth['info']['nickname'] || omniauth['info']['name'] }
68 66 
     end
69 67 
   end
70 68

+ 8 - 1
config/initializers/devise.rb
Просмотреть файл 

@@ -136,7 +136,7 @@ Devise.setup do |config|
136 136 
   # The time you want to timeout the user session without activity. After this
137 137 
   # time the user will be asked for credentials again. Default is 30 minutes.
138 138 
   # config.timeout_in = 30.minutes
139 
-
139 
+
140 140 
   # If true, expires auth token on session timeout.
141 141 
   # config.expire_auth_token_on_timeout = false
142 142 
 
@@ -213,6 +213,7 @@ Devise.setup do |config|
213 213 
   # Add a new OmniAuth provider. Check the wiki for more information on setting
214 214 
   # up on your models and hooks.
215 215 
   # config.omniauth :github, 'APP_ID', 'APP_SECRET', :scope => 'user,public_repo'
216 
+
216 217 
   if defined?(OmniAuth::Strategies::Twitter) &&
217 218 
      (key = ENV["TWITTER_OAUTH_KEY"]).present? &&
218 219 
      (secret = ENV["TWITTER_OAUTH_SECRET"]).present?
 
@@ -237,6 +238,12 @@ Devise.setup do |config|
237 238 
     config.omniauth :github, key, secret
238 239 
   end
239 240
241 
+  if defined?(OmniAuth::Strategies::Dropbox) &&
242 
+     (key = ENV["DROPBOX_OAUTH_KEY"]).present? &&
243 
+     (secret = ENV["DROPBOX_OAUTH_SECRET"]).present?
244 
+    config.omniauth :dropbox, key, secret
245 
+  end
246 
+
240 247 
   # ==> Warden configuration
241 248 
   # If you want to use other strategies, that are not supported by Devise, or
242 249 
   # change the failure app, you can configure them inside the config.warden block.

+ 1 - 0
config/locales/devise.en.yml
Просмотреть файл 

@@ -54,6 +54,7 @@ en:
54 54 
       tumblr: 'Tumblr'
55 55 
       github: 'GitHub'
56 56 
       37signals: '37Signals (Basecamp)'
57 
+      dropbox: 'Dropbox'
57 58 
     mailer:
58 59 
       confirmation_instructions:
59 60 
         subject: 'Confirmation instructions'

+ 2 - 0
spec/env.test
Просмотреть файл 

@@ -5,4 +5,6 @@ TUMBLR_OAUTH_KEY=tumblroauthsecret
5 5 
 TUMBLR_OAUTH_SECRET=tumblroauthsecret
6 6 
 THIRTY_SEVEN_SIGNALS_OAUTH_KEY=TESTKEY
7 7 
 THIRTY_SEVEN_SIGNALS_OAUTH_SECRET=TESTSECRET
8 
+DROPBOX_OAUTH_KEY=dropboxoauthkey
9 
+DROPBOX_OAUTH_SECRET=dropboxoauthsecret
8 10 
 FAILED_JOBS_TO_KEEP=2

+ 173 - 0
spec/models/agents/dropbox_watch_agent_spec.rb
Просмотреть файл 

@@ -0,0 +1,173 @@
1 
+require 'spec_helper'
2 
+
3 
+describe Agents::DropboxWatchAgent do
4 
+  before(:each) do
5 
+    @agent = Agents::DropboxWatchAgent.new(
6 
+      name: 'save to dropbox',
7 
+      options: {
8 
+        access_token: '70k3n',
9 
+        dir_to_watch: '/my/dropbox/dir',
10 
+        expected_update_period_in_days: 2
11 
+      }
12 
+    )
13 
+    @agent.user = users(:bob)
14 
+    @agent.service = services(:generic)
15 
+    @agent.save!
16 
+  end
17 
+
18 
+  it 'cannot receive events' do
19 
+    expect(@agent.cannot_receive_events?).to eq true
20 
+  end
21 
+
22 
+  it 'has agent description' do
23 
+    expect(@agent.description).to_not be_nil
24 
+  end
25 
+
26 
+  it 'has event description' do
27 
+    expect(@agent.event_description).to_not be_nil
28 
+  end
29 
+
30 
+  describe '#valid?' do
31 
+    before(:each) { expect(@agent.valid?).to eq true }
32 
+
33 
+    it 'requires a "dir_to_watch"' do
34 
+      @agent.options[:dir_to_watch] = nil
35 
+      expect(@agent.valid?).to eq false
36 
+    end
37 
+
38 
+    describe 'expected_update_period_in_days' do
39 
+      it 'needs to be present' do
40 
+        @agent.options[:expected_update_period_in_days] = nil
41 
+        expect(@agent.valid?).to eq false
42 
+      end
43 
+
44 
+      it 'needs to be a positive integer' do
45 
+        @agent.options[:expected_update_period_in_days] = -1
46 
+        expect(@agent.valid?).to eq false
47 
+      end
48 
+    end
49 
+  end
50 
+
51 
+  describe '#check' do
52 
+
53 
+    let(:first_result) { [{ 'path' => '1.json', 'rev' => '1', 'modified' => '01-01-01' }] }
54 
+
55 
+    before(:each) do
56 
+      stub.proxy(Dropbox::API::Client).new do |api|
57 
+        stub(api).ls('/my/dropbox/dir') { first_result }
58 
+      end
59 
+    end
60 
+
61 
+    it 'saves the directory listing in its memory' do
62 
+      @agent.check
63 
+      expect(@agent.memory).to eq 'contents' => first_result
64 
+    end
65 
+
66 
+    context 'first time' do
67 
+
68 
+      before(:each) { @agent.memory = {} }
69 
+
70 
+      it 'does not send any events' do
71 
+        expect { @agent.check }.to_not change(Event, :count)
72 
+      end
73 
+
74 
+    end
75 
+
76 
+    context 'subsequent calls' do
77 
+
78 
+      let(:second_result) { [{ 'path' => '2.json', 'rev' => '1', 'modified' => '02-02-02' }] }
79 
+
80 
+      before(:each) do
81 
+        @agent.memory = { 'contents' => 'not_empty' }
82 
+
83 
+        stub.proxy(Dropbox::API::Client).new do |api|
84 
+          stub(api).ls('/my/dropbox/dir') { second_result }
85 
+        end
86 
+      end
87 
+
88 
+      it 'sends an event upon a different directory listing' do
89 
+        payload = { 'diff' => 'object as hash' }
90 
+        stub.proxy(Agents::DropboxWatchAgent::DropboxDirDiff).new(@agent.memory['contents'], second_result) do |diff|
91 
+          stub(diff).empty? { false }
92 
+          stub(diff).to_hash { payload }
93 
+        end
94 
+        expect { @agent.check }.to change(Event, :count).by(1)
95 
+        expect(Event.last.payload).to eq(payload)
96 
+      end
97 
+
98 
+      it 'does not sent any events when there is no difference on the directory listing' do
99 
+        stub.proxy(Agents::DropboxWatchAgent::DropboxDirDiff).new(@agent.memory['contents'], second_result) do |diff|
100 
+          stub(diff).empty? { true }
101 
+        end
102 
+
103 
+        expect { @agent.check }.to_not change(Event, :count)
104 
+      end
105 
+
106 
+    end
107 
+  end
108 
+
109 
+  describe Agents::DropboxWatchAgent::DropboxDirDiff do
110 
+
111 
+    let(:previous) { [
112 
+      { 'path' => '1.json', 'rev' => '1' },
113 
+      { 'path' => '2.json', 'rev' => '1' },
114 
+      { 'path' => '3.json', 'rev' => '1' }
115 
+    ] }
116 
+
117 
+    let(:current) { [
118 
+      { 'path' => '1.json', 'rev' => '2' },
119 
+      { 'path' => '3.json', 'rev' => '1' },
120 
+      { 'path' => '4.json', 'rev' => '1' }
121 
+    ] }
122 
+
123 
+    describe '#empty?' do
124 
+
125 
+      it 'is true when no differences are detected' do
126 
+        diff = Agents::DropboxWatchAgent::DropboxDirDiff.new(previous, previous)
127 
+        expect(diff.empty?).to eq true
128 
+      end
129 
+
130 
+      it 'is false when differences were detected' do
131 
+        diff = Agents::DropboxWatchAgent::DropboxDirDiff.new(previous, current)
132 
+        expect(diff.empty?).to eq false
133 
+      end
134 
+
135 
+    end
136 
+
137 
+    describe '#to_hash' do
138 
+
139 
+      subject(:diff_hash) { Agents::DropboxWatchAgent::DropboxDirDiff.new(previous, current).to_hash }
140 
+
141 
+      it 'detects additions' do
142 
+        expect(diff_hash[:added]).to eq [{ 'path' => '4.json', 'rev' => '1' }]
143 
+      end
144 
+
145 
+      it 'detects removals' do
146 
+        expect(diff_hash[:removed]).to eq [ { 'path' => '2.json', 'rev' => '1' } ]
147 
+      end
148 
+
149 
+      it 'detects updates' do
150 
+        expect(diff_hash[:updated]).to eq [ { 'path' => '1.json', 'rev' => '2' } ]
151 
+      end
152 
+
153 
+      context 'when the previous value is not defined' do
154 
+        it 'considers all additions' do
155 
+          diff_hash = Agents::DropboxWatchAgent::DropboxDirDiff.new(nil, current).to_hash
156 
+          expect(diff_hash[:added]).to eq current
157 
+          expect(diff_hash[:removed]).to eq []
158 
+          expect(diff_hash[:updated]).to eq []
159 
+        end
160 
+      end
161 
+
162 
+      context 'when the current value is not defined' do
163 
+        it 'considers all removals' do
164 
+          diff_hash = Agents::DropboxWatchAgent::DropboxDirDiff.new(previous, nil).to_hash
165 
+          expect(diff_hash[:added]).to eq []
166 
+          expect(diff_hash[:removed]).to eq previous
167 
+          expect(diff_hash[:updated]).to eq []
168 
+        end
169 
+      end
170 
+    end
171 
+  end
172 
+
173 
+end